import Core.Gadget as Gadget
import datetime
from Analysis import General, Plot, PreProcess, IndustryAnalysis
import pandas as pd
import matplotlib.pyplot as plt


def PlotFactorSeries(database, symbol, factorName, extrafilters = {}):
    filter = {}
    #filter={"StdDateTime":{"$gte": datetime1, "$lte": datetime2}}
    filter["Symbol"] = symbol
    #filter["Params"] = "Daily"
    sort = [("DateTime", 1)]
    dataSeries = database.Find("Factor", factorName, filter, sort)

    plotData = []
    x = []
    for data in dataSeries:
        #plotData.append([data["StdDateTime"],data["Value"]])
        plotData.append(data["Value"])
        x.append(data["DateTime"])

    plt.plot(x, plotData)
    plt.ylabel(factorName)
    #plt.grid(True, linestyle="-.", color="r", linewidth="3")
    plt.grid(True)
    plt.show()


def PlotFactor(database, factorName):

    # ---不做时间限制，数据量应该很大---
    data = []
    favtorValues = database.Find("Factor", factorName)
    for value in favtorValues:
        data.append([value["DateTime"], value["Value"]])
    df = pd.DataFrame(data, columns=["Symbol", factorName])
    print(df.describe())

    # PreProcess.Outlier_NSigma(df)
    PreProcess.Outlier_Percentile(df)
    print("After Clear Outliers")
    print(df.describe())

    Plot.PlotHistgram(df, factorName)
    Plot.PrintQuantile(df, factorName, axis=0)


def PlotFactorProfile(database, factorName, reportDate, datetimeField="ReportDate", plot=True):
    #
    print(factorName, reportDate)

    # ---Build Data---
    factorName = factorName
    # df = General.Profile2(database, [factorName], reportDate, datetimeField)
    df = General.Profile(database, reportDate, [factorName])

    print(df.describe())

    # Sorting
    df.sort_values(by=factorName, ascending=True, inplace=True)

    if plot:
        print(df.head(10))
        print("...")
        print(df.tail(10))
        # print(df)

    # df[factorName].plot.density()
    # plt.show()

    # print()
    d = reportDate
    if not isinstance(reportDate, datetime.date):
        d = reportDate.date()
    data = [d]
    for i in range(11):
        value = df[factorName].quantile(i * 0.1)
        # print(i * 10, "%", value)
        data.append(value)
    #
    print()
    # print(" 5%", df[factorName].quantile(0.05))
    # print("95%", df[factorName].quantile(0.95))

    #
    data.append(df[factorName].quantile(0.01))
    data.append(df[factorName].quantile(0.05))
    data.append(df[factorName].quantile(0.95))
    data.append(df[factorName].quantile(0.99))
    data.append(df[factorName].quantile(0.25))
    data.append(df[factorName].quantile(0.75))

    #
    data.append(df[factorName].shape[0])

    # ---Plot---
    if plot:
        Plot.PlotHistgram(df, factorName)
        # Plot.PrintQuantile(df, factorName, axis=1)

    #
    floor = df.quantile(0.05).values[0]
    celling = df.quantile(0.95).values[0]
    df = df[df[factorName] > floor]
    df = df[df[factorName] < celling]

    print("After Cutting Tails")
    print(df.describe())

    return data


# 查看某年公司排名情况
def TopCompany(factorName, datetime1):
    factors = [factorName]
    df = General.Profile2(database, factors, datetime1)
    #
    dfSorted = df.sort_values(by=factorName, ascending=False)
    print(dfSorted)


# ---Plot Single Factor History---
def FactorHistory(symbol, factorName):
    df = General.FactorDataSeries(database, symbol, factorName)
    df.set_index("ReportDate", inplace=True)
    print(df)
    General.PlotData(df, symbol)


# ValuationFactor 按照月份进行Plot
def PlotFactorProfileHistory(datetime1, datetime2, factorName,
                             filterNegtive=False, filterFunc=None, valuationFactor=True, params={}):

    # 先确定遍历时间
    datetimes = []
    if valuationFactor: # 估值因子按月
        filter = {"Symbol": "000001.SZ"}
        filter["StdDateTime"] = {"$gte": datetime1, "$lte": datetime2}
        bm = database.Find("Factor", factorName, filter, sort=[("StdDateTime", 1)])
        for b in bm:
            datetimes.append(b["DateTime"])
    else: # 其他因子按照报告期
        datetimes = Gadget.GenerateReportDates(datetime1,datetime2)

    profileData = []
    fields = []

    # ---Loop Factors---
    for dt in datetimes:
        #
        factorName = factorName
        # df = Common.LoadFactorProfile(database, [factorName], dt)

        filter = {}
        if valuationFactor:
            date = Gadget.ToDate(dt)
            filter["Date"] = date
        else:
            filter["ReportDate"] = dt

        # 准备剖面数据
        favtorValues = database.Find("Factor", factorName, filter)
        data = []
        for v in favtorValues:
            params["Symbol"] = v["Symbol"]
            params["Value"] = v["Value"]
            if filterNegtive:
                if v["Value"] < 0:
                    continue
            if filterFunc != None:
                if not filterFunc(params):
                    continue
            data.append([v["Symbol"], v["Value"]])
        #
        df = pd.DataFrame(data, columns=["Symbol", factorName])

        # 剖面数据再提取 Profile Feature
        #
        entry = [] # 数据
        entry.append(Gadget.ToDate(dt)) # datetime
        entry.append(df.shape[0]) # count
        #
        fields = [] # Header
        fields.append("DateTime")
        fields.append("Count")

        # percentile
        for i in range(11):
            value = df[factorName].quantile(i * 0.1)
            fields.append(str(i*10) + "%")
            entry.append(value)
            if i == 0:
                entry.append(df[factorName].quantile(0.05))
                fields.append(str(5) + "%")
            if i == 10 - 1:
                entry.append(df[factorName].quantile(0.95))
                fields.append(str(95) + "%")

        #
        profileData.append(entry)

        # ---Print---
        strLine = ""
        for e in entry:
            strLine += str(e) + ","
        print(strLine)
    #
    df = pd.DataFrame(profileData, columns=fields)
    return df


def IndustryDeterminFilter(params):
    #
    instrumentBySymbol = params["InstrumentBySymbol"]
    symbol = params["Symbol"]
    targetIndustry = params["TargetIndustry"]
    industryField = params["IndustryField"]

    #
    instrument = instrumentBySymbol[symbol]
    industry = instrument[industryField]

    #
    if industry == targetIndustry:
        return True
    return False


# 分行业plot
def FactorProfileHistory_BySector(datetime1, datetime2, factorName, valuationFactor=False):
    #
    industryField = "SWIndustry1"
    #
    instrumentBySymbol = {}
    industries = []
    instruments = database.Find("Instruments", "Stock")
    for instrument in instruments:
        instrumentBySymbol[instrument["Symbol"]] = instrument
        industry = instrument[industryField]
        if industry not in industries:
            if industry == None:
                continue
            industries.append(industry)

    #
    params = {}
    params["InstrumentBySymbol"] = instrumentBySymbol
    params["IndustryField"] = industryField
    #
    folderName = "d:/data/IndustryAnalysis/" + factorName
    Gadget.CreateFolder(folderName)
    for industry in industries:
        print("Process", industry)
        params["TargetIndustry"] = industry
        df = FactorProfileHistory(datetime1, datetime2, factorName,
                                  valuationFactor=valuationFactor,
                                  filterNegtive=False,
                                  filterFunc=IndustryDeterminFilter,
                                  params=params)
        df.to_csv(folderName + "/" + factorName + "_" + industry + ".csv")


# 兼容不同时间周期
def PlotFactorProfileHistory2(database, datetime1, datetime2, factorName, dateTimeField="DateTime"):
    #
    filter = {}
    filter["Symbol"] = "000001.SZ"
    filter[dateTimeField] = {"$gte": datetime1, "$lte": datetime2}
    factorValues = database.Find("Factor", factorName, filter)

    #
    data = []
    for factorValue in factorValues:
        curDateTime = factorValue[dateTimeField]
        d = PlotFactorProfile(database, factorName, curDateTime, dateTimeField, plot=False)
        data.append(d)

    #
    df = pd.DataFrame(data, columns=["DateTime",
                                     "0","10","20","30","40","50","60","70","80","90","100",
                                     "1","5","95","99","25","75",
                                     "Count"])
    #
    print(df)
    ax1 = plt.subplot(311)
    # df["25"].plot(ax=ax1)
    # df["50"].plot(ax=ax1)
    # df["75"].plot(ax=ax1)
    ax1.set_ylabel('Percentile')
    df.plot(x="DateTime", y=["25", "50", "75"], grid=True, ax=ax1)

    #
    ax2 = plt.subplot(312)
    # df["1"].plot(ax=ax2)
    # df["5"].plot(ax=ax2)
    # df["25"].plot(ax=ax2)
    # df["50"].plot(ax=ax2)
    # df["75"].plot(ax=ax2)
    # df["95"].plot(ax=ax2)
    # df["99"].plot(ax=ax2)
    # ax1.set_xlabel('DateTime')
    df.plot(x="DateTime", y=["1", "5", "25", "50", "75", "95", "99"], grid=True, ax=ax2)

    #
    # ax2 = plt.subplot(212, sharex=ax1)
    ax3 = plt.subplot(313)
    df["Count"].plot(ax=ax3)
    ax3.set_ylabel('Count')
    #
    plt.tight_layout()
    #
    plt.show()
    #
    return df


if __name__ == '__main__':

    #
    # factorName = "AssetTurnoverLYR"
    # factorName = "AssetTurnoverTTM"
    # factorName = "BookToMarket"

    # factorName = "EarningToPrice_TTM"
    # factorName = "EarningToPrice_LYR"
    # factorName = "EnterpriseMultiple_TTM"
    # factorName = "EnterpriseMultiple_R_LYR"
    # factorName = "GrossProfitMarginLYR"
    factorName = "EarningToPrice_TTM"

    # factorName = "Leverage"

    # factorName = "PriceEarningNetIncomeTTM"
    # factorName = "ProfitMarginOPTTM"
    # factorName = "PriceBookLF"
    # factorName = "PriceEarningOPTTM"
    # factorName = "PriceEarningOPTTM"
    # factorName = "PE_OperatingProfit_TTM"
    # factorName = "PE_NetIncome2_TTM"
    # factorName = "PCF_LYR"
    # factorName = "PCF_R_LYR"
    # factorName = "PCF_R_TTM"
    # factorName = "ROE_NetIncome2_TTM"
    # factorName = "PB_LF"
    # factorName = "PS_TTM"
    # factorName = "PS_LYR"
    # factorName = "PE_NetIncome2_TTM"
    # factorName = "PE_NetIncome2_LYR"

    # factorName = "ROEOperatingProfitTTM"
    # factorName = "EnterpriseMultiple_R_LYR"

    #
    from Core.Config import *
    cfgPathFilename = os.getcwd() + "/../config.json"
    config = Config(cfgPathFilename)
    database = config.DataBase("MySQL")
    realtime = config.RealTime()

    # ---全部数据---
    # 发现分布，发现离群值
    # PlotFactor(factorName)

    # ---某个时间点---
    datetime1 = datetime.datetime(2017, 12, 31)
    # df = PlotFactorProfile(database, factorName, datetime1)

    # ---随历史变化的（剖面）分位数---
    datetime1 = datetime.datetime(2005,1,1)
    datetime2 = datetime.datetime(2019,6,1)
    #
    df = PlotFactorProfileHistory2(database, datetime1, datetime2, factorName)
    # df.to_csv("D:/Data/FactorCheck/" + factorName + "_" + Gadget.ToDateString(datetime2) + ".csv")
    #
    df = PlotFactorProfileHistory(database, datetime1, datetime2, factorName, valuationFactor=False)
    # FactorProfileHistory_BySector(datetime1, datetime2, factorName, valuationFactor=False)

    # ---查看某年公司排名情况---
    datetime1 = datetime.datetime(2005, 12, 31)
    datetime2 = datetime.datetime(2012, 12, 31)
    datetime3 = datetime.datetime(2018, 9, 30)
    # TopCompany(factorName, datetime1)

    # ---查看极值公司---
    symbol = "000622.SZ"
    symbol = "600745.SH"
    symbol = "600016.SH"
    symbol = "000002.SZ"
    symbol = "601988.SH"
    symbol = "601398.SH"
    symbol = "600822.SH"
    symbol = "000626.SZ"
    symbol = "000001.SZ"
    # FactorHistory(symbol, factorName)
